import math


def read_tag(path):
    tagmap = {}
    tags = []
    with open(path,'r', encoding='utf-8') as f:
        for line in f:
            if line not in tagmap:
                tagmap[line] = len(tagmap)
            tags.append(tagmap[line])
    return tags


def storeIDF_for_svm(path, tag = None , prewordmap = None):
    wordmap = {}
    if prewordmap is not None:
        wordmap = prewordmap
    idf = {}
    f = open(path, 'r',encoding='utf-8')
    docCount = 0
    for line in f:
        docCount += 1
        lines = line.split()
        tmp = set()
        for item in lines:
            if prewordmap is None and item not in wordmap:
                wordmap[item] = len(wordmap)
            if item not in wordmap:
                continue

            if wordmap[item] not in tmp:
                tmp.add(wordmap[item])
                if wordmap[item] not in idf:
                    idf[wordmap[item]] = 0
                idf[wordmap[item]] += 1

    f.close()

    f = open(path, 'r',encoding='utf-8')
    fw = open(path + ".idf_svm", 'w',encoding='utf-8')
    # fw.write(' '.join(["c_"+str(i) for i in range(len(wordmap))]))
    # fw.write(" class\n")

    for index,line in enumerate(f):
        if index % 100==0:
            print(index)
        lines = line.split()
        wline = [0 for item in wordmap]
        for item in lines:
            if item not in wordmap:
                continue
            wline[wordmap[item]] += 1
        kline = []
        if tag != None:
            fw.write("c_"+str(tag[index])+" ")
        for i in range(len(wordmap)):
            if i not in idf or wline[i]==0:
                pass
                # kline.append(str(i+1)+":0")
            else:
                kline.append(str(i+1)+":"+str((float)(wline[i]) / len(lines) * math.log(docCount / idf[i])))
        fw.write(' '.join(kline))

        fw.write("\n")
    if tag != None:
        print(index, len(tag))
        assert index+1 == len(tag)
    fw.close()
    f.close()
    return wordmap


def storeIDF(path):
    wordmap = {}
    idf = {}
    f = open(path, 'r')
    docCount = 0
    for line in f:
        docCount += 1
        lines = line.split()
        tmp = set()
        for item in lines:
            if item not in wordmap:
                wordmap[item] = len(wordmap)
            if item not in wordmap:
                continue

            if wordmap[item] not in tmp:
                tmp.add(wordmap[item])
                if wordmap[item] not in idf:
                    idf[wordmap[item]] = 0
                idf[wordmap[item]] += 1

    f.close()

    f = open(path, 'r')
    fw = open(path + ".idf", 'w')
    # fw.write(' '.join(["c_"+str(i) for i in range(len(wordmap))]))
    # fw.write(" class\n")

    for index,line in enumerate(f):
        if index % 100==0:
            print(index)
        lines = line.split()
        wline = [0 for item in wordmap]
        for item in lines:
            if item not in wordmap:
                continue
            wline[wordmap[item]] += 1
        kline = []

        for i in range(len(wordmap)):
            if i not in idf:
                kline.append("0")
            else:
                kline.append(str((float)(wline[i]) / len(lines) * math.log(docCount / idf[i])))
        fw.write(' '.join(kline))

        fw.write("\n")

    fw.close()
    f.close()
    return wordmap

def get_svm(path):
    with open(path, 'r', encoding='utf-8') as f,open(path+".svm", 'w', encoding='utf-8') as w:
        for index, line in enumerate(f):
            maps = {}
            lines = line.split()
            for item in lines:
                if item not in maps:
                    maps[item] = 0
                maps[item]+=1
            w.write(str(index+1)+" "+str(len(lines))+" "+"1")
            for key in maps:
                w.write(" "+key+":"+str(maps[key]))
            w.write("\n")


if __name__ =="__main__":
    root = "G:/intellij/TopicModelForShortText/My_LDA/data/new-tweet/new-tweet.data"
    # storeIDF(root )
    # storeIDF_for_svm(root+"new-tweet.data",tag = read_tag(root+"new-tweet.tag"))
    get_svm(root)